annex.addunlocked support for tree imports

Honor annex.addunlocked configuration when importing a tree from a special
remote.

Note, in a --no-content import, the object file will not be populated
(usually) and so expressions that match on mime type will not match. Tested
this and it works ok, the file just ends up locked. Updated docs for the
mime expressions to mention that they can't match when the file is present

Note that in Command.Sync.pullThirdPartyPopulated, recordImportTree is
called without a AddUnlockedMatcher. Since the tree generated here is not
exposed to the user and does not contain usual filenames, there is no need
of the overhead of checking it.

diff --git a/Annex/Import.hs b/Annex/Import.hs
index 3cc068692cae3b8ce8eca3a545a0bab2db2d649c..587d866a96aa7802fbc23af6f50633e203d57110 100644 (file)
--- a/Annex/Import.hs
+++ b/Annex/Import.hs
@@ -107,9 +107,10 @@ buildImportCommit
        :: Remote
        -> ImportTreeConfig
        -> ImportCommitConfig
+       -> AddUnlockedMatcher
        -> Imported
        -> Annex (Maybe Ref)
-buildImportCommit remote importtreeconfig importcommitconfig imported =
+buildImportCommit remote importtreeconfig importcommitconfig addunlockedmatcher imported =
        case importCommitTracking importcommitconfig of
                Nothing -> go Nothing
                Just trackingcommit -> inRepo (Git.Ref.tree trackingcommit) >>= \case
@@ -117,7 +118,7 @@ buildImportCommit remote importtreeconfig importcommitconfig imported =
                        Just _ -> go (Just trackingcommit)
   where
        go trackingcommit = do
-               (importedtree, updatestate) <- recordImportTree remote importtreeconfig imported
+               (importedtree, updatestate) <- recordImportTree remote importtreeconfig (Just addunlockedmatcher) imported
                buildImportCommit' remote importcommitconfig trackingcommit importedtree >>= \case
                        Just finalcommit -> do
                                updatestate
@@ -132,10 +133,11 @@ buildImportCommit remote importtreeconfig importcommitconfig imported =
 recordImportTree
        :: Remote
        -> ImportTreeConfig
+       -> Maybe AddUnlockedMatcher
        -> Imported
        -> Annex (History Sha, Annex ())
-recordImportTree remote importtreeconfig imported = do
-       importedtree@(History finaltree _) <- buildImportTrees basetree subdir imported
+recordImportTree remote importtreeconfig addunlockedmatcher imported = do
+       importedtree@(History finaltree _) <- buildImportTrees basetree subdir addunlockedmatcher imported
        return (importedtree, updatestate finaltree)
   where
        basetree = case importtreeconfig of
@@ -177,7 +179,7 @@ recordImportTree remote importtreeconfig imported = do
                        }
                return oldexport
 
-       -- downloadImport takes care of updating the location log
+       -- importKeys takes care of updating the location log
        -- for the local repo when keys are downloaded, and also updates
        -- the location log for the remote for keys that are present in it.
        -- That leaves updating the location log for the remote for keys
@@ -283,11 +285,12 @@ buildImportCommit' remote importcommitconfig mtrackingcommit imported@(History t
 buildImportTrees
        :: Ref
        -> Maybe TopFilePath
+       -> Maybe AddUnlockedMatcher
        -> Imported
        -> Annex (History Sha)
-buildImportTrees basetree msubdir (ImportedFull imported) = 
-       buildImportTreesGeneric convertImportTree basetree msubdir imported
-buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) imported) = do
+buildImportTrees basetree msubdir addunlockedmatcher (ImportedFull imported) = 
+       buildImportTreesGeneric (convertImportTree addunlockedmatcher) basetree msubdir imported
+buildImportTrees basetree msubdir addunlockedmatcher (ImportedDiff (LastImportedTree oldtree) imported) = do
        importtree <- if null (importableContents imported)
                then pure oldtree
                else applydiff
@@ -312,7 +315,7 @@ buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) impor
                        oldtree
        
        mktreeitem (loc, DiffChanged v) = 
-               Just <$> mkImportTreeItem msubdir loc v
+               Just <$> mkImportTreeItem addunlockedmatcher msubdir loc v
        mktreeitem (_, DiffRemoved) = 
                pure Nothing
 
@@ -320,17 +323,26 @@ buildImportTrees basetree msubdir (ImportedDiff (LastImportedTree oldtree) impor
                
        isremoved (_, v) = v == DiffRemoved
 
-convertImportTree :: Maybe TopFilePath -> [(ImportLocation, Either Sha Key)] -> Annex Tree
-convertImportTree msubdir ls = 
-       treeItemsToTree <$> mapM (uncurry $ mkImportTreeItem msubdir) ls
+convertImportTree :: Maybe AddUnlockedMatcher -> Maybe TopFilePath -> [(ImportLocation, Either Sha Key)] -> Annex Tree
+convertImportTree maddunlockedmatcher msubdir ls = 
+       treeItemsToTree <$> mapM (uncurry $ mkImportTreeItem maddunlockedmatcher msubdir) ls
 
-mkImportTreeItem :: Maybe TopFilePath -> ImportLocation -> Either Sha Key -> Annex TreeItem
-mkImportTreeItem msubdir loc v = case v of
-       Right k -> do
-               relf <- fromRepo $ fromTopFilePath topf
-               symlink <- calcRepo $ gitAnnexLink relf k
-               linksha <- hashSymlink symlink
-               return $ TreeItem treepath (fromTreeItemType TreeSymlink) linksha
+mkImportTreeItem :: Maybe AddUnlockedMatcher -> Maybe TopFilePath -> ImportLocation -> Either Sha Key -> Annex TreeItem
+mkImportTreeItem maddunlockedmatcher msubdir loc v = case v of
+       Right k -> case maddunlockedmatcher of
+               Nothing -> mklink k
+               Just addunlockedmatcher -> do
+                       objfile <- calcRepo (gitAnnexLocation k)
+                       let mi = MatchingFile FileInfo
+                               { contentFile = objfile
+                               , matchFile = getTopFilePath topf
+                               , matchKey = Just k
+                               }
+                       ifM (checkAddUnlockedMatcher NoLiveUpdate addunlockedmatcher mi)
+                               ( mkpointer k
+                               , mklink k
+                               )
+                               
        Left sha -> 
                return $ TreeItem treepath (fromTreeItemType TreeFile) sha
   where
@@ -338,6 +350,13 @@ mkImportTreeItem msubdir loc v = case v of
        treepath = asTopFilePath lf
        topf = asTopFilePath $
                maybe lf (\sd -> getTopFilePath sd P.</> lf) msubdir
+       mklink k = do
+               relf <- fromRepo $ fromTopFilePath topf
+               symlink <- calcRepo $ gitAnnexLink relf k
+               linksha <- hashSymlink symlink
+               return $ TreeItem treepath (fromTreeItemType TreeSymlink) linksha
+       mkpointer k = TreeItem treepath (fromTreeItemType TreeFile)
+               <$> hashPointerFile k
 
 {- Builds a history of git trees using ContentIdentifiers.
  -
@@ -604,8 +623,8 @@ getLastImportedTree remote = do
  - generates Keys without downloading.
  -
  - Generates either a Key or a git Sha, depending on annex.largefiles.
- - But when importcontent is False, it cannot match on annex.largefiles
- - (or generate a git Sha), so always generates Keys.
+ - But when importcontent is False, it cannot generate a git Sha, 
+ - so always generates Keys.
  -
  - Supports concurrency when enabled.
  -

diff --git a/CHANGELOG b/CHANGELOG
index 0e2f177b0bf64fae8dd83122935c4625851f07cd..11d27b8c86c96c36c6bac3a3f22d897203f494f1 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,8 @@ git-annex (10.20241203) UNRELEASED; urgency=medium
     default unset behavior.
   * sync: Avoid misleading warning about future preferred content
     transition when preferred content is set to "".
+  * Honor annex.addunlocked configuration when importing a tree from a
+    special remote.
 
  -- Joey Hess <id@joeyh.name>  Mon, 02 Dec 2024 13:41:31 -0400
 

diff --git a/Command/Import.hs b/Command/Import.hs
index f06543bb7ebfdab56cc5387244949223e05aa722..c35055927eafb5f07787a0b8f9068e54d64efdc5 100644 (file)
--- a/Command/Import.hs
+++ b/Command/Import.hs
@@ -147,8 +147,10 @@ seek o@(RemoteImportOptions {}) = startConcurrency commandStages $ do
                (pure Nothing)
                (Just <$$> inRepo . toTopFilePath . toRawFilePath)
                (importToSubDir o)
+       addunlockedmatcher <- addUnlockedMatcher
        seekRemote r (importToBranch o) subdir (importContent o) 
                (checkGitIgnoreOption o)
+               addunlockedmatcher
                (messageOption o)
 
 startLocal :: ImportOptions -> AddUnlockedMatcher -> GetFileMatcher -> DuplicateMode -> (RawFilePath, RawFilePath) -> CommandStart
@@ -322,8 +324,8 @@ verifyExisting key destfile (yes, no) = do
        verifyEnoughCopiesToDrop [] key Nothing Nothing needcopies mincopies [] preverified tocheck
                (const yes) no
 
-seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> [String] -> CommandSeek
-seekRemote remote branch msubdir importcontent ci importmessages = do
+seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> AddUnlockedMatcher -> [String] -> CommandSeek
+seekRemote remote branch msubdir importcontent ci addunlockedmatcher importmessages = do
        importtreeconfig <- case msubdir of
                Nothing -> return ImportTree
                Just subdir ->
@@ -337,7 +339,7 @@ seekRemote remote branch msubdir importcontent ci importmessages = do
        trackingcommit <- fromtrackingbranch Git.Ref.sha
        cmode <- annexCommitMode <$> Annex.getGitConfig
        let importcommitconfig = ImportCommitConfig trackingcommit cmode importmessages'
-       let commitimport = commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig
+       let commitimport = commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig addunlockedmatcher
 
        importabletvar <- liftIO $ newTVarIO Nothing
        void $ includeCommandAction (listContents remote importtreeconfig ci importabletvar)
@@ -383,10 +385,10 @@ listContents' remote importtreeconfig ci a =
                        , err
                        ]
 
-commitRemote :: Remote -> Branch -> RemoteTrackingBranch -> Maybe Sha -> ImportTreeConfig -> ImportCommitConfig -> Imported -> CommandStart
-commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig imported =
+commitRemote :: Remote -> Branch -> RemoteTrackingBranch -> Maybe Sha -> ImportTreeConfig -> ImportCommitConfig -> AddUnlockedMatcher -> Imported -> CommandStart
+commitRemote remote branch tb trackingcommit importtreeconfig importcommitconfig addunlockedmatcher imported =
        starting "update" ai si $ do
-               importcommit <- buildImportCommit remote importtreeconfig importcommitconfig imported
+               importcommit <- buildImportCommit remote importtreeconfig importcommitconfig addunlockedmatcher imported
                next $ updateremotetrackingbranch importcommit
   where
        ai = ActionItemOther (Just $ UnquotedString $ fromRef $ fromRemoteTrackingBranch tb)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index c9436778bde4808edb7ef2919e6ace622b26dfb8..5b2fa3c3800ed0dfc2b76fc99ceb356114200ee8 100644 (file)
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -53,6 +53,7 @@ import Annex.Path
 import Annex.Wanted
 import Annex.Content
 import Annex.WorkTree
+import Annex.FileMatcher
 import Command.Get (getKey')
 import qualified Command.Move
 import qualified Command.Export
@@ -77,7 +78,6 @@ import Annex.CurrentBranch
 import Annex.Import
 import Annex.CheckIgnore
 import Annex.PidLock
-import Types.FileMatcher
 import Types.GitConfig
 import Types.Availability
 import qualified Database.Export as Export
@@ -580,7 +580,8 @@ importRemote importcontent o remote currbranch
                        let (branch, subdir) = splitRemoteAnnexTrackingBranchSubdir b
                        if canImportKeys remote importcontent
                                then do
-                                       Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True) []
+                                       addunlockedmatcher <- addUnlockedMatcher
+                                       Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True) addunlockedmatcher []
                                        -- Importing generates a branch
                                        -- that is not initially connected
                                        -- to the current branch, so allow
@@ -607,7 +608,7 @@ pullThirdPartyPopulated o remote
   where
        go (Just importable) = importChanges remote ImportTree False True importable >>= \case
                ImportFinished imported -> do
-                       (_t, updatestate) <- recordImportTree remote ImportTree imported
+                       (_t, updatestate) <- recordImportTree remote ImportTree Nothing imported
                        next $ do
                                updatestate
                                return True

diff --git a/doc/bugs/addunlocked_true_is_not_in_effect_for_import.mdwn b/doc/bugs/addunlocked_true_is_not_in_effect_for_import.mdwn
index 9814cca7f48295e741a66718db5013fb5202a42f..b46e1838282dd806078d14353eb1bb07e7fac9c1 100644 (file)
--- a/doc/bugs/addunlocked_true_is_not_in_effect_for_import.mdwn
+++ b/doc/bugs/addunlocked_true_is_not_in_effect_for_import.mdwn
@@ -54,3 +54,5 @@ This was to consider using `import` for a folder with DANDI stats. For now I wil
 
 [[!meta author=yoh]]
 [[!tag projects/dandi]]
+
+> [[fixed|done]] --[[Joey]]

diff --git a/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_1_d8dc7bb0e6d3ae4e77a2d17ef672b3ab._comment b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_1_d8dc7bb0e6d3ae4e77a2d17ef672b3ab._comment
new file mode 100644 (file)
index 0000000..88619dd
--- /dev/null
+++ b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_1_d8dc7bb0e6d3ae4e77a2d17ef672b3ab._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-12-18T19:24:38Z"
+ content="""
+Turns out that while `git-annex import` from a directory does support
+addunlocked, this was forgotten about when implementing the newer special
+remote tree import.
+
+I agree that this should be supported.
+"""]]

diff --git a/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_2_40371f91f89cc035c8edc1b719210a5c._comment b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_2_40371f91f89cc035c8edc1b719210a5c._comment
new file mode 100644 (file)
index 0000000..c64e3f7
--- /dev/null
+++ b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_2_40371f91f89cc035c8edc1b719210a5c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2024-12-19T15:31:59Z"
+ content="""
+Note that for --no-content imports, it will not be possible for mimetype=
+and mimeencoding= expressions to match.
+
+So if addunlocked is set to such an expression, it will not match and will
+add the file locked. Does not seem like a blocker.
+"""]]

diff --git a/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_3_4640377996674e1330d542354b36ef3c._comment b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_3_4640377996674e1330d542354b36ef3c._comment
new file mode 100644 (file)
index 0000000..61d4d28
--- /dev/null
+++ b/doc/bugs/addunlocked_true_is_not_in_effect_for_import/comment_3_4640377996674e1330d542354b36ef3c._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-12-19T15:34:48Z"
+ content="""
+Implemented this.
+"""]]

diff --git a/doc/git-annex-matching-expression.mdwn b/doc/git-annex-matching-expression.mdwn
index 246196b1114f3a00568d7eda27b7e7b31e4934a6..dda5efb1d8f451854230c0e5c3fa91d7697d4d4e 100644 (file)
--- a/doc/git-annex-matching-expression.mdwn
+++ b/doc/git-annex-matching-expression.mdwn
@@ -48,6 +48,10 @@ The following terms can be used:
 
   The MIME types are the same that are displayed by running `file --mime-type`
 
+  This only matches when the content of the file is present in the local
+  repository. Usually that is the case, but eg, when importing from a
+  special remote with --no-content, the content is usually not present.
+
   This is only available to use when git-annex was built with the
   MagicMime build flag.
 
@@ -60,6 +64,10 @@ The following terms can be used:
 
   The MIME encodings are the same that are displayed by running `file --mime-encoding`
 
+  This only matches when the content of the file is present in the local
+  repository. Usually that is the case, but eg, when importing from a
+  special remote with --no-content, the content is usually not present.
+
   This is only available to use when git-annex was built with the
   MagicMime build flag.